
#ifndef PTX_INTRINSICS_H_INCLUDED
#define PTX_INTRINSICS_H_INCLUDED

#ifndef NL
#   define NL "\n"
#endif

#define ptx_dot_v2(type, d, a, b) \
    NL "    mul"type" "d", "a".x, "b".x;" \
    NL "    fma.rn"type" "d", "a".y, "b".y, "d";"

#define ptx_dot_v3(type, d, a, b) \
    NL "    mul"type" "d", "a".x, "b".x;" \
    NL "    fma.rn"type" "d", "a".y, "b".y, "d";" \
    NL "    fma.rn"type" "d", "a".z, "b".z, "d";"

#define ptx_dot_v4(type, d, a, b) \
    NL "    mul"type" "d", "a".x, "b".x;" \
    NL "    fma.rn"type" "d", "a".y, "b".y, "d";" \
    NL "    fma.rn"type" "d", "a".z, "b".z, "d";" \
    NL "    fma.rn"type" "d", "a".w, "b".w, "d";"

#define ptx_dot_sat_v2(type, d, a, b) \
    NL "    mul"type" "d", "a".x, "b".x;" \
    NL "    fma.rn.sat"type" "d", "a".y, "b".y, "d";"

#define ptx_dot_sat_v3(type, d, a, b) \
    NL "    mul"type" "d", "a".x, "b".x;" \
    NL "    fma.rn"type" "d", "a".y, "b".y, "d";" \
    NL "    fma.rn.sat"type" "d", "a".z, "b".z, "d";"

#define ptx_dot_sat_v4(type, d, a, b) \
    NL "    mul"type" "d", "a".x, "b".x;" \
    NL "    fma.rn"type" "d", "a".y, "b".y, "d";" \
    NL "    fma.rn"type" "d", "a".z, "b".z, "d";" \
    NL "    fma.rn.sat"type" "d", "a".w, "b".w, "d";"

#define ptx_mul_m2_v2(type, dv, m, v) \
    NL "    /* "dv" = "m" * "v"; */ {" \
    NL "    .reg .v2 "type" t;" \
    NL "    mov.v2.f32 t, "v";" \
            ptx_dot_v2(type, dv".x", m"0", "t") \
            ptx_dot_v2(type, dv".y", m"1", "t") \
    NL "    }"

#define ptx_mul_m3_v3(type, dv, m, v) \
    NL "    /* "dv" = "m" * "v"; */ {" \
    NL "    .reg .v4 "type" t;" \
    NL "    mov.v4.f32 t, "v";" \
            ptx_dot_v3(type, dv".x", m"0", "t") \
            ptx_dot_v3(type, dv".y", m"1", "t") \
            ptx_dot_v3(type, dv".z", m"2", "t") \
    NL "    }"

#define ptx_mul_m4_v4(type, dv, m, v) \
    NL "    /* "dv" = "m" * "v"; */ {" \
    NL "    .reg .v4 "type" t;" \
    NL "    mov.v4.f32 t, "v";" \
            ptx_dot_v4(type, dv".x", m"0", "t") \
            ptx_dot_v4(type, dv".y", m"1", "t") \
            ptx_dot_v4(type, dv".z", m"2", "t") \
            ptx_dot_v4(type, dv".w", m"3", "t") \
    NL "    }"

// 6 fma + 6 mul
#define ptx_mul_m2_m2(type, dm, ma, mb) \
    NL "    /* "dm" = "ma" * "mb"; */ {" \
    NL "    .reg .v2 "type" t<2>;" \
    NL "    mul"type" t0.x, "ma"0.x, "mb"0.x;" \
    NL "    fma.rn"type" t0.x, "ma"0.y, "mb"1.x, t0.x;" \
    NL "    mul"type" t0.y, "ma"0.x, "mb"0.y;" \
    NL "    fma.rn"type" t0.y, "ma"0.y, "mb"1.y, t0.y;" \
    NL "    mul"type" t1.x, "ma"1.x, "mb"0.x;" \
    NL "    fma.rn"type" t1.x, "ma"1.y, "mb"1.x, t1.x;" \
    NL "    mul"type" t1.y, "ma"1.x, "mb"0.y;" \
    NL "    fma.rn"type" t1.y, "ma"1.y, "mb"1.y, t1.y;" \
    NL "    mul"type" t2.x, "ma"2.x, "mb"0.x;" \
    NL "    fma.rn"type" t2.x, "ma"2.y, "mb"1.x, t2.x;" \
    NL "    mul"type" t2.y, "ma"2.x, "mb"0.y;" \
    NL "    fma.rn"type" t2.y, "ma"2.y, "mb"1.y, t2.y;" \
    NL "    mov.v2"type" "dm"0, t0;" \
    NL "    mov.v2"type" "dm"1, t1;" \
    NL "    }"

// 18 fma + 9 mul
#define ptx_mul_m3_m3(type, dm, ma, mb) \
    NL "    /* "dm" = "ma" * "mb"; */ {" \
    NL "    .reg .v4 "type" t<3>;" \
    NL "    mul"type" t0.x, "ma"0.x, "mb"0.x;" \
    NL "    fma.rn"type" t0.x, "ma"0.y, "mb"1.x, t0.x;" \
    NL "    fma.rn"type" t0.x, "ma"0.z, "mb"2.x, t0.x;" \
    NL "    mul"type" t0.y, "ma"0.x, "mb"0.y;" \
    NL "    fma.rn"type" t0.y, "ma"0.y, "mb"1.y, t0.y;" \
    NL "    fma.rn"type" t0.y, "ma"0.z, "mb"2.y, t0.y;" \
    NL "    mul"type" t0.z, "ma"0.x, "mb"0.z;" \
    NL "    fma.rn"type" t0.z, "ma"0.y, "mb"1.z, t0.z;" \
    NL "    fma.rn"type" t0.z, "ma"0.z, "mb"2.z, t0.z;" \
    NL "    mul"type" t1.x, "ma"1.x, "mb"0.x;" \
    NL "    fma.rn"type" t1.x, "ma"1.y, "mb"1.x, t1.x;" \
    NL "    fma.rn"type" t1.x, "ma"1.z, "mb"2.x, t1.x;" \
    NL "    mul"type" t1.y, "ma"1.x, "mb"0.y;" \
    NL "    fma.rn"type" t1.y, "ma"1.y, "mb"1.y, t1.y;" \
    NL "    fma.rn"type" t1.y, "ma"1.z, "mb"2.y, t1.y;" \
    NL "    mul"type" t1.z, "ma"1.x, "mb"0.z;" \
    NL "    fma.rn"type" t1.z, "ma"1.y, "mb"1.z, t1.z;" \
    NL "    fma.rn"type" t1.z, "ma"1.z, "mb"2.z, t1.z;" \
    NL "    mul"type" t2.x, "ma"2.x, "mb"0.x;" \
    NL "    fma.rn"type" t2.x, "ma"2.y, "mb"1.x, t2.x;" \
    NL "    fma.rn"type" t2.x, "ma"2.z, "mb"2.x, t2.x;" \
    NL "    mul"type" t2.y, "ma"2.x, "mb"0.y;" \
    NL "    fma.rn"type" t2.y, "ma"2.y, "mb"1.y, t2.y;" \
    NL "    fma.rn"type" t2.y, "ma"2.z, "mb"2.y, t2.y;" \
    NL "    mul"type" t2.z, "ma"2.x, "mb"0.z;" \
    NL "    fma.rn"type" t2.z, "ma"2.y, "mb"1.z, t2.z;" \
    NL "    fma.rn"type" t2.z, "ma"2.z, "mb"2.z, t2.z;" \
    NL "    mov.v4"type" "dm"0, t0;" \
    NL "    mov.v4"type" "dm"1, t1;" \
    NL "    mov.v4"type" "dm"2, t2;" \
    NL "    }"

// 48 fma + 16 mul
#define ptx_mul_m4_m4(type, dm, ma, mb) \
    NL "    /* "dm" = "ma" * "mb"; */ {" \
    NL "    .reg .v4 "type" t<4>;" \
    NL "    mul"type" t0.x, "ma"0.x, "mb"0.x;" \
    NL "    fma.rn"type" t0.x, "ma"0.y, "mb"1.x, t0.x;" \
    NL "    fma.rn"type" t0.x, "ma"0.z, "mb"2.x, t0.x;" \
    NL "    fma.rn"type" t0.x, "ma"0.w, "mb"3.x, t0.x;" \
    NL "    mul"type" t0.y, "ma"0.x, "mb"0.y;" \
    NL "    fma.rn"type" t0.y, "ma"0.y, "mb"1.y, t0.y;" \
    NL "    fma.rn"type" t0.y, "ma"0.z, "mb"2.y, t0.y;" \
    NL "    fma.rn"type" t0.y, "ma"0.w, "mb"3.y, t0.y;" \
    NL "    mul"type" t0.z, "ma"0.x, "mb"0.z;" \
    NL "    fma.rn"type" t0.z, "ma"0.y, "mb"1.z, t0.z;" \
    NL "    fma.rn"type" t0.z, "ma"0.z, "mb"2.z, t0.z;" \
    NL "    fma.rn"type" t0.z, "ma"0.w, "mb"3.z, t0.z;" \
    NL "    mul"type" t0.w, "ma"0.x, "mb"0.w;" \
    NL "    fma.rn"type" t0.w, "ma"0.y, "mb"1.w, t0.w;" \
    NL "    fma.rn"type" t0.w, "ma"0.z, "mb"2.w, t0.w;" \
    NL "    fma.rn"type" t0.w, "ma"0.w, "mb"3.w, t0.w;" \
    NL "    mul"type" t1.x, "ma"1.x, "mb"0.x;" \
    NL "    fma.rn"type" t1.x, "ma"1.y, "mb"1.x, t1.x;" \
    NL "    fma.rn"type" t1.x, "ma"1.z, "mb"2.x, t1.x;" \
    NL "    fma.rn"type" t1.x, "ma"1.w, "mb"3.x, t1.x;" \
    NL "    mul"type" t1.y, "ma"1.x, "mb"0.y;" \
    NL "    fma.rn"type" t1.y, "ma"1.y, "mb"1.y, t1.y;" \
    NL "    fma.rn"type" t1.y, "ma"1.z, "mb"2.y, t1.y;" \
    NL "    fma.rn"type" t1.y, "ma"1.w, "mb"3.y, t1.y;" \
    NL "    mul"type" t1.z, "ma"1.x, "mb"0.z;" \
    NL "    fma.rn"type" t1.z, "ma"1.y, "mb"1.z, t1.z;" \
    NL "    fma.rn"type" t1.z, "ma"1.z, "mb"2.z, t1.z;" \
    NL "    fma.rn"type" t1.z, "ma"1.w, "mb"3.z, t1.z;" \
    NL "    mul"type" t1.w, "ma"1.x, "mb"0.w;" \
    NL "    fma.rn"type" t1.w, "ma"1.y, "mb"1.w, t1.w;" \
    NL "    fma.rn"type" t1.w, "ma"1.z, "mb"2.w, t1.w;" \
    NL "    fma.rn"type" t1.w, "ma"1.w, "mb"3.w, t1.w;" \
    NL "    mul"type" t2.x, "ma"2.x, "mb"0.x;" \
    NL "    fma.rn"type" t2.x, "ma"2.y, "mb"1.x, t2.x;" \
    NL "    fma.rn"type" t2.x, "ma"2.z, "mb"2.x, t2.x;" \
    NL "    fma.rn"type" t2.x, "ma"2.w, "mb"3.x, t2.x;" \
    NL "    mul"type" t2.y, "ma"2.x, "mb"0.y;" \
    NL "    fma.rn"type" t2.y, "ma"2.y, "mb"1.y, t2.y;" \
    NL "    fma.rn"type" t2.y, "ma"2.z, "mb"2.y, t2.y;" \
    NL "    fma.rn"type" t2.y, "ma"2.w, "mb"3.y, t2.y;" \
    NL "    mul"type" t2.z, "ma"2.x, "mb"0.z;" \
    NL "    fma.rn"type" t2.z, "ma"2.y, "mb"1.z, t2.z;" \
    NL "    fma.rn"type" t2.z, "ma"2.z, "mb"2.z, t2.z;" \
    NL "    fma.rn"type" t2.z, "ma"2.w, "mb"3.z, t2.z;" \
    NL "    mul"type" t2.w, "ma"2.x, "mb"0.w;" \
    NL "    fma.rn"type" t2.w, "ma"2.y, "mb"1.w, t2.w;" \
    NL "    fma.rn"type" t2.w, "ma"2.z, "mb"2.w, t2.w;" \
    NL "    fma.rn"type" t2.w, "ma"2.w, "mb"3.w, t2.w;" \
    NL "    mul"type" t3.x, "ma"3.x, "mb"0.x;" \
    NL "    fma.rn"type" t3.x, "ma"3.y, "mb"1.x, t3.x;" \
    NL "    fma.rn"type" t3.x, "ma"3.z, "mb"2.x, t3.x;" \
    NL "    fma.rn"type" t3.x, "ma"3.w, "mb"3.x, t3.x;" \
    NL "    mul"type" t3.y, "ma"3.x, "mb"0.y;" \
    NL "    fma.rn"type" t3.y, "ma"3.y, "mb"1.y, t3.y;" \
    NL "    fma.rn"type" t3.y, "ma"3.z, "mb"2.y, t3.y;" \
    NL "    fma.rn"type" t3.y, "ma"3.w, "mb"3.y, t3.y;" \
    NL "    mul"type" t3.z, "ma"3.x, "mb"0.z;" \
    NL "    fma.rn"type" t3.z, "ma"3.y, "mb"1.z, t3.z;" \
    NL "    fma.rn"type" t3.z, "ma"3.z, "mb"2.z, t3.z;" \
    NL "    fma.rn"type" t3.z, "ma"3.w, "mb"3.z, t3.z;" \
    NL "    mul"type" t3.w, "ma"3.x, "mb"0.w;" \
    NL "    fma.rn"type" t3.w, "ma"3.y, "mb"1.w, t3.w;" \
    NL "    fma.rn"type" t3.w, "ma"3.z, "mb"2.w, t3.w;" \
    NL "    fma.rn"type" t3.w, "ma"3.w, "mb"3.w, t3.w;" \
    NL "    mov.v4"type" "dm"0, t0;" \
    NL "    mov.v4"type" "dm"1, t1;" \
    NL "    mov.v4"type" "dm"2, t2;" \
    NL "    mov.v4"type" "dm"3, t3;" \
    NL "    }"

// v2
#define ptx_op_v2(op, d, v) \
    NL "    "op" "d".x, "v".x;" \
    NL "    "op" "d".y, "v".y;"

#define ptx_op_v2_s(op, d, v, s) \
    NL "    "op" "d".x, "v".x, "s";" \
    NL "    "op" "d".y, "v".y, "s";"

#define ptx_op_v2_v2(op, d, v0, v1) \
    NL "    "op" "d".x, "v0".x, "v1".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y;"

#define ptx_op_v2_s_v2(op, d, v0, s, v1) \
    NL "    "op" "d".x, "v0".x, "s", "v1".x;" \
    NL "    "op" "d".y, "v0".y, "s", "v1".y;"

#define ptx_op_v2_v2_s(op, d, v0, v1, s) \
    NL "    "op" "d".x, "v0".x, "v1".x, "s";" \
    NL "    "op" "d".y, "v0".y, "v1".y, "s";"

#define ptx_op_v2_v2_v2(op, d, v0, v1, v2) \
    NL "    "op" "d".x, "v0".x, "v1".x, "v2".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y, "v2".y;"

// v3
#define ptx_op_v3(op, d, v) \
    NL "    "op" "d".x, "v".x;" \
    NL "    "op" "d".y, "v".y;" \
    NL "    "op" "d".z, "v".z;"

#define ptx_op_v3_s(op, d, v, s) \
    NL "    "op" "d".x, "v".x, "s";" \
    NL "    "op" "d".y, "v".y, "s";" \
    NL "    "op" "d".z, "v".z, "s";"

#define ptx_op_v3_v3(op, d, v0, v1) \
    NL "    "op" "d".x, "v0".x, "v1".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z;"

#define ptx_op_v3_s_v3(op, d, v0, s, v1) \
    NL "    "op" "d".x, "v0".x, "s", "v1".x;" \
    NL "    "op" "d".y, "v0".y, "s", "v1".y;" \
    NL "    "op" "d".z, "v0".z, "s", "v1".z;"

#define ptx_op_v3_v3_s(op, d, v0, v1, s) \
    NL "    "op" "d".x, "v0".x, "v1".x, "s";" \
    NL "    "op" "d".y, "v0".y, "v1".y, "s";" \
    NL "    "op" "d".z, "v0".z, "v1".z, "s";" \

#define ptx_op_v3_v3_v3(op, d, v0, v1, v2) \
    NL "    "op" "d".x, "v0".x, "v1".x, "v2".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y, "v2".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z, "v2".z;"

// v4
#define ptx_op_v4(op, d, v) \
    NL "    "op" "d".x, "v".x;" \
    NL "    "op" "d".y, "v".y;" \
    NL "    "op" "d".z, "v".z;" \
    NL "    "op" "d".w, "v".w;"

#define ptx_op_v4_s(op, d, v, s) \
    NL "    "op" "d".x, "v".x, "s";" \
    NL "    "op" "d".y, "v".y, "s";" \
    NL "    "op" "d".z, "v".z, "s";" \
    NL "    "op" "d".w, "v".w, "s";"

#define ptx_op_v4_v4(op, d, v0, v1) \
    NL "    "op" "d".x, "v0".x, "v1".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z;" \
    NL "    "op" "d".w, "v0".w, "v1".w;"

#define ptx_op_v4_s_v4(op, d, v0, s, v1) \
    NL "    "op" "d".x, "v0".x, "s", "v1".x;" \
    NL "    "op" "d".y, "v0".y, "s", "v1".y;" \
    NL "    "op" "d".z, "v0".z, "s", "v1".z;" \
    NL "    "op" "d".w, "v0".w, "s", "v1".w;"

#define ptx_op_v4_v4_s(op, d, v0, v1, s) \
    NL "    "op" "d".x, "v0".x, "v1".x, "s";" \
    NL "    "op" "d".y, "v0".y, "v1".y, "s";" \
    NL "    "op" "d".z, "v0".z, "v1".z, "s";" \
    NL "    "op" "d".w, "v0".w, "v1".w, "s";"

#define ptx_op_v4_v4_v4(op, d, v0, v1, v2) \
    NL "    "op" "d".x, "v0".x, "v1".x, "v2".x;" \
    NL "    "op" "d".y, "v0".y, "v1".y, "v2".y;" \
    NL "    "op" "d".z, "v0".z, "v1".z, "v2".z;" \
    NL "    "op" "d".w, "v0".w, "v1".w, "v2".w;"

#endif
